{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Univariate Feature Selection\n",
    "\n",
    "\n",
    "An example showing univariate feature selection.\n",
    "\n",
    "Noisy (non informative) features are added to the iris data and\n",
    "univariate feature selection is applied. For each feature, we plot the\n",
    "p-values for the univariate feature selection and the corresponding\n",
    "weights of an SVM. We can see that univariate feature selection\n",
    "selects the informative features and that these have larger SVM weights.\n",
    "\n",
    "In the total set of features, only the 4 first ones are significant. We\n",
    "can see that they have the highest score with univariate feature\n",
    "selection. The SVM assigns a large weight to one of these features, but also\n",
    "Selects many of the non-informative features.\n",
    "Applying univariate feature selection before the SVM\n",
    "increases the SVM weight attributed to the significant features, and will\n",
    "thus improve classification.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import datasets, svm\n",
    "from sklearn.feature_selection import SelectPercentile, f_classif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #############################################################################\n",
    "# Import some data to play with\n",
    "\n",
    "# The iris dataset\n",
    "iris = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some noisy data not correlated\n",
    "E = np.random.uniform(0, 0.1, size=(len(iris.data), 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the noisy data to the informative features\n",
    "X = np.hstack((iris.data, E))\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAEWCAYAAABYGk2QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3hU1b3/8ffXEIEIyk3Bcos3EAIkyEVQiaEqXsAj+MMCVStaWxVaqxYLtD0SbXvqqaiUHi+11UrViop4qaXUA4IFi1ZuCgIei6JBkZvcIXL7/v7YO9NJMoEEkixMPq/nmWf27Nl7rbX3zHxmz5o9a8zdERGR6ndU6AaIiNRWCmARkUAUwCIigSiARUQCUQCLiASiABYRCUQBLBViZlea2atVVHZ7M1tkZtvM7OaqqONIY2bDzWxuFZRbZY+TVB4FcCBm9k0zm29m281sjZn91czOCd2ug3H3p9y9XxUV/yNgtrs3dPeJh1OQmc02s+srqV1HNDPLNDM3szpF86r4cZJKogAOwMxuAyYA/wU0B9oADwKXhWzXwSS/wKtIW+C9Kq6jXKphW0XA3XWpxgtwHLAduOIAy9QlCujP4ssEoG58Xx6wmuhocR2wBhgIXAL8H/AF8OOksvKBKcAzwDZgIZCddP8YYGV83zJgUNJ9w4E3gPvjcn8ez5ubtIwDNwIfAJuABwCL70sD7gU2AB8B34uXr5Nim18D9gGF8f5pF++H8cAnwFrgYaB+vHxj4BVgfVzvK0Cr+L5flCjrf4DMknUDs4Hry9rWeP51wPK4jr8Bbct4zOoBTwIbgc3A20DzpMf80fix+jTej2lJ9Sbvz9OB/43b8D7wjaT76sf782NgCzA3nvdJvG3b40vvFOWeFbdpS3x9Von98LN4+7cBrwLNQr9WasMleANq2wW4CNibKoSSlrkLeBM4ATge+Afws/i+vHj9O4B04DtxCP0JaAhkxcFzcrx8PrAHGBwvP4ooDNPj+68Avkb0aWgIsAM4Mb5veFzX94E68Yu95AvbicKvEdGR/Hrgovi+G4lCvRVRYM6gjACOl59NHIjx7QnAy0CTeNv+DPwyvq8p8P+AjPi+54AXD1BWZsm6KR3AJbd1IPAvoEM876fAP8po+w1x+zKI3ni6AcfG970I/BY4Jn5M/wnckFTv3Hj6GKAAuDau7wyiN6+s+P4H4ja3jOs4i+hNKtW2JZfbhOgN5Oq43GHx7aZJ+2El0Zte/fj23aFfK7XhErwBte0CXAl8fpBlVgKXJN2+EFgVT+cBu/j3EVTD+MV3ZtLyC4CB8XQ+8GbSfUcRHYn1KaPuxcBl8fRw4JMS9yde2PFtB85Juv0sMCaefq0oaOLb55cMihJlz+bfgWhEbwanJN3fG/iojHVzgE2pyopvpwqp5PpSbetfgW+X2Hc7SXEUTHSk/A+gS4n5zYEviY/c43nDgFkl9yfRG+CcEuv/FhgX172LpE8vB9m25HKvBv5ZYp15wPCk/fDTpPtGANNDv1Zqw0X9XNVvI9DMzOq4+94ylvka0cfMIh/H8xJluPu+eHpXfL026f5dQIOk2wVFE+6+38xWF5VnZt8CbiN6EROv1yzVugfwedL0zqS6v1Zi/fKUVeR4oqPJBWZWNM+Ijvwwswyi7oKLiI6uARqaWVrSvqmoku1rC/zazO5NmmdER6Afl1j2CaA1MNnMGhF1R/wkLiMdWJO0HUelqKuovjPNbHPSvDpx2c2IujlWVnCboPTzifh2y6TbZT2GUoX0JVz1m0fURTDwAMt8RvRiLNImnneoWhdNmNlRRF0Cn5lZW+B3RH2zTd29EbCUKGSKHM5weWviukq1oxw2EL2RZLl7o/hynLsXBcMPgfZER/7HArnx/KK2l2z3jvg6I2leixLLlFyngOgIvlHSpb67/6NkY919j7vf6e4diboGBgDfisv4kqhPtaiMY909K8U2FwCvl6ivgbvfFO+PQuCUFOsd7DEq+XyC6Dn16UHWkyqmAK5m7r6FqP/2ATMbaGYZZpZuZheb2a/ixZ4Gfmpmx5tZs3j5Jw+j2m5mdnn8zf4tRIHwJlGfoxP122Jm1wKdDqOekp4FfmBmLeOjwtHlXdHd9xO9OdxvZifE7WtpZhfGizQkCujNZtaE6GN6srXAyUnlrScKnKvMLM3MriN1mCV7GBhrZllx/ceZ2RWpFjSzvmbW2czSgK1E/e773H0N0Zda95rZsWZ2lJmdYmbnpijmFaCdmV0dPyfSzayHmXWI98djwH1m9rV4G3qbWV2ix29/8vaWMC0u95tmVsfMhgAd4/okIAVwAO5+H9HH/p8SvXgKiI5CX4wX+TkwH3gXWEJ05sLPD6PKl4j6F4u+iLk8PmJbRvSt+jyiwOpM9E14ZfkdUfi8CywiCoK9RGcolMdooi/B3jSzrURf4rWP75tA9IXRBqI3k+kl1v01MNjMNplZ0TnF3wFuJ+oGyiLqsy2Tu78A/DdRt8JWok8HF5exeAuis022Ep018Tr/ftP8FnA00ReSm+LlTkxR3zagHzCU6Kj187j+uvEio4ieD28TnSXx38BR7r6T6MyPN8xss5n1KlHuRqIj8h/G2/4jYIC7bzjQ9kvVKzpdSGooM8sHTnX3q46AtlwMPOzuJT8Oi9RKOgKWKmNm9c3skvhjb0uiboIXQrdL5EihAJaqZMCdRB+7FxF9NL8jaItEjiDqghARCURHwCIigVTohxjNmjXzzMzMKmqKiEjNtGDBgg3ufnzJ+RUK4MzMTObPn195rRIRqQXMrOQvEQF1QYiIBKMAFhEJRAEsIhKIRkOTKrNnzx5Wr15NYWFh6KaIVIt69erRqlUr0tPTy7W8AliqzOrVq2nYsCGZmZkkDcUoUiO5Oxs3bmT16tWcdNJJ5VpHXRBSZQoLC2natKnCV2oFM6Np06YV+sSnAJYqpfCV2qSiz3cFsIhIIApgEZFAalwAZ7ZugZlhZqSl1Y2u69dPzGvRpk3oJoqIADUwgD9evRYfDz4e9u/fDeSzv7AQZs2CWbNYW1CR/4WUypT85lgZl8zWJf/SrbRVq1bRqVPxf1nKz89n/PjxB1zvrLPOOqxtrUhZmzdv5sEHH6y0+qrDrl27OPfcc9m3r3x/bvLb3/6WkSNHVnu9lWH37t3k5uayd+/eYtOVQaehSbUpenOsLDZq7cEXOkT/+McB/62oXIr+evxgZRUF8IgRIw67zkNV1NajjirfMdljjz3G5ZdfTlpaWrmWf/fdd+ncufPhNPGQ6q0MRx99NOeddx7PPPMMV155ZbHpw1XjjoBFymvVqlV06NCB73znO2RlZdGvXz927doFQIMG0Z8vjx49utjRaX5+PvfeG/1L/cCBA+nWrRtZWVk88sgjxcocMWIEZ5xxBgUFBYmyylpnzJgxrFy5kpycHG6//XYAnnzySXr27ElOTg433HBDqSO+HTt20L9/f7Kzs+nUqRPPPPMMAH/84x/p0qUL2dnZXH311Ynl77vvPjp16kSnTp2YMGFCmW09WL1FnnrqKS677LJy7+slS5aUCuAVK1aQm5tLVlYW559/Phs2RH9Rt3z5cnJzc+nSpQv33HMPp556apn1Dh06lCFDhnDmmWfStm1b/vKXv5S7TRUxcOBAnnrqqVLTh63ona88l27duvmRDnAfH10Ah/zoetas6AKhm1hrLFu2rNjt5MemMi7leSw/+ugjz8rKKjZv3Lhxfs899/hHH33kaWlpvmjRInd3v+KKK/yJJ55wd/djjjnG3d0XLlzoubm5iXU7dOjgH3/8sbu7b9y40d3dd+7c6VlZWb5hwwb/6KOP3Mx83rx5iXWKyjrQOsltXLZsmQ8YMMB3797t7u433XSTT5o0qdg2TJkyxa+//vrE7c2bN/vSpUu9Xbt2vn79+mJ1zZ8/3zt16uTbt2/3bdu2eceOHX3hwoWl2lqeet3dv/zyS2/evHmZ+zyVxo0b++bNmxO3CwsLE+1wd7/77rv9xz/+se/Zs8e7du2amH/jjTf6ZZddVma9p59+uo8ZM8bd3efMmeM9evSoULvKa+/evd6sWbNS06mUfN67uwPzPUWm6ghYarSyzsssmn/SSSeRk5MDQLdu3Vi1alWx5bp27cq6dev47LPPeOedd2jcuDFt4i9yJ06cSHZ2Nr169aKgoIAPPvgAgLZt29KrV7E/Jk4oa51kM2fOZMGCBfTo0YOcnBxmzpzJhx9+WGyZzp07M2PGDEaPHs2cOXM47rjjeO211xg8eDDNmjUDoEmTJgDMnTuXQYMGccwxx9CgQQMuv/xy5syZU6qt5akXYMOGDTRq1Chx+/zzz08cXSdfXnrpJQAKCgpo2LAhxx13XGKdF198kXPOOYeuXbsC0LFjR9atW8fUqVPJzs4uNj87Oztlvbt27WLDhg2MGzcuseymTZtS7vdUhg8fXu5l09LSOProo9m2bVux6cOlPmCp0Zo2bVrqRfnFF18kfipat27dxPy0tLREF0SywYMHM2XKFD7//HOGDh0KwOzZs5kxYwbz5s0jIyODvLy8xC+gjjnmmJRtOdA6ydyda665hl/+8pdlble7du1YsGAB06ZNY+zYsfTr149GjRqlfMPxA/ztWHJby1MvQP369Yu1e8aMGQdcPlX/77Jly4rNW7JkCR07duTdd99NvCECLF26lIsuuihlvUuXLuW0006jXr16ACxcuJDs7GwefPBBOnbsSF5eHtdddx0TJ07k3nvv5YsvvqBRo0bceeed7Ny5k4KCAsaPH8+oUaMYMWIE48ePJz09nXHjxrFz5052795drPvpyy+/TNSVPH04dAQsNVqDBg048cQTmTlzJhCF7/Tp0znnnHPKXcbQoUOZPHkyU6ZMYfDgwQBs2bKFxo0bk5GRwYoVK3jzzTcPWk5Z6zRs2LDY0dR5553HlClTWLduXaLNH39cfDzvzz77jIyMDK666ipGjRrFwoULOe+883j22WfZuHFjYj2A3NxcXnzxRXbu3MmOHTt44YUX6NOnT6n2ladegMaNG7Nv375y/+Q2Vf9vy5YtWbZsGQAffvghTzzxBN/61rdo2rQp//d//wfA4sWLefLJJxNHwCXrfeedd/jkk08oLCxkx44djBs3jltvvZXOnTvz3nvv8fe//52ePXuyZcsW9uzZQ6NGjRL7fOHCheTm5ibK3rlzJxkZGTzyyCPs2rWLRo0asX379kR7N27cyPHHH096enqx6cOlI2CpNm1bNa/UMxfatmperuX++Mc/MnLkSH74wx8CMG7cOE455ZRS3Q1lycrKYtu2bbRs2ZITTzwRgIsuuoiHH36YLl260L59+zK7HJKVtU7Tpk05++yz6dSpExdffDH33HMPP//5z+nXrx/79+8nPT2dBx54gLZt2ybKWrJkCbfffjtHHXUU6enpPPTQQ2RlZfGTn/yEc889l7S0NLp27crjjz/OGWecwfDhw+nZsycA119/PV27di21/R07djxovUX69evH3LlzOf/88w+63UuWLGH69Ok8/fTTAJx44om89tprTJs2jc6dO1O/fn0ee+wxmjZtytVXX03//v3p0aMHvXv3JjMzk5NPPjllve+88w5XXnkleXl5bN26lR//+MecffbZbN68mcmTJzN//nx+97vf8d3vfpdf//rXrF+/noL4NNS3336bxYsXc+ONN7J169bEJ4dFixbxwAMPFPtkBDBr1iwuueSSUtOHLVXHcFkXfQknFZHqywipGRYuXOhXXXVVpZe7bdu2xPSvfvUr/8lPflJmvX369PEVK1akLCcrK8unTp3q7u7jx4/3O+64w0eOHOkTJ050d/dvf/vb/vvf/95HjRrlY8aM8bvvvtvd3V9++WUfOnSo33777f7Xv/41Ud6gQYMSdSVPp1KRL+F0BCwiFda1a1f69u3Lvn37KvWc3Pvvv5/JkyeTnp7O2WefzX333VdmvStXruS0005LWc7SpUsT00WffJL9/ve/T7nepZdeyqWXXlps3u7duxk4cCDt27cvNl0ZzA/QQV9S9+7d/Uj/U04zS5zsb6MA8qPLrFnRzL59D/ilhFSe5cuX06FDh9DNEKlWqZ73ZrbA3buXXFZfwomIBKIAFhEJRAFM8UFiyjPAi4hIZdCXcBQfJKYqB3gREUmmI2ARkUAUwCIigSiApdq0aNGqUgdkb9Gi1UHr/MUvfkFWVhZdunQhJyeHt956i/z8fMaOHVtsucWLFydOHcrMzCz1U92cnJxSA7sfrksuuYTNmzcfcJm8vDxSnfq5ePFipk2bVqntkeqnPmCpNmvXfkp0XnZllXfgsubNm8crr7zCwoULqVu3Lhs2bGD37t0MGzaMiy++uNigM5MnT+ab3/xm4va2bdsoKCigdevWLF++vNLanOxwAnTx4sXMnz+/8n4SK0HoCFhqrDVr1tCsWbPE7/qbNWvG1772Ndq3b0+jRo146623Ess+++yziZHOAL7xjW8kBjl/+umnGTZsWMo6RowYwcsvvwzAoEGDuO666wB49NFH+elPfwqUPbh6ZmZmYhDyn/3sZ5x++ulccMEFDBs2rNhfJj333HP07NmTdu3aMWfOHHbv3s0dd9zBM888Q05OTqKd8tWjAJYaq1+/fhQUFNCuXTtGjBjB66+/nrhv2LBhTJ48GYA333yTpk2bFvtZ6+DBg5k6dSoAf/7zn0v9PLVIbm5uYmzdTz/9NDHC19y5c+nTpw/Lly/nmWee4Y033mDx4sWkpaWV+jeF+fPn8/zzz7No0SKmTp1aqsth7969/POf/2TChAnceeedHH300dx1110MGTKExYsXM2TIkMPcUxKKAlhqrAYNGrBgwQIeeeQRjj/+eIYMGcLjjz8ORENMTpkyhf379zN58uRSR7hNmjShcePGTJ48mQ4dOpCRkZGyjj59+jBnzhyWLVtGx44dad68OWvWrGHevHmcddZZ5RrkfO7cuVx22WXUr1+fhg0blgr7yy+/HEg9YLx8takPWGq0tLQ08vLyyMvLo3PnzkyaNInhw4fTunVrMjMzef3113n++eeZN29eqXWHDBnCyJEjE6GdSsuWLdm0aRPTp08nNzeXL774gmeffZYGDRrQsGHDcg1yfrCxSYq6UNLS0irt33jlyKAjYKmx3n///WJ/+bN48eJiY9sOGzaMW2+9lVNOOYVWrUqfUTFo0CB+9KMfceGFFx6wnt69ezNhwgRyc3Pp06cP48ePT5xFUZ5Bzs855xz+/Oc/U1hYyPbt28v1x5IlB3GXryYdAUu1ad685UHPXKhoeQeyfft2vv/977N582bq1KnDqaeemvgnYoArrriCH/zgB/zmN79JuX7Dhg0ZPXr0QdvRp08fXn31VU499VTatm3LF198kQjg8gxy3qNHD/7jP/6D7Oxs2rZtS/fu3Yv9f1oqffv25e677yYnJ4exY8eqH/grSsNRplhHw1VWDg1HWX7bt2+nQYMG7Ny5k9zcXB555BHOOOOM0M2SQ1CR4Sh1BCxyBPjud7/LsmXLKCws5JprrlH41hIKYJEjwJ/+9KfQTZAA9CWciEggCmARkUAUwCIigSiARUQCUQBLtWnRpk3lDkfZps1B69RwlAf33HPP0aFDB/r27RtkmMvhw4czZcqUCq+3atWqYl9ezp8/n5tvvrkym1bldBaEVJu1BQX/Ph+7Msrr2/eA92s4yvJ59NFHefDBB+nbty+PP/54hcvdu3cvdepUf5QUBXDR49a9e3e6dy91qu0RTUfAUmPVpuEoX3/9dXJycsjJyaFr164pf6Y8cOBAunXrRlZWVuIXgXfddRdz587lxhtv5NZbby1V7o4dO7juuuvo0aMHXbt25aWXXgLg8ccf54orruDSSy+lX79+xerZsWMH/fv3Jzs7m06dOiX244IFCzj33HPp1q0bF154IWvWrCnVxrKW+de//sX5559PdnY2Z5xxBitXrmTMmDHMmTOHnJwc7r//fmbPns2AAQOA6CffAwcOpEuXLvTq1Yt3330XgPz8fK677jry8vI4+eSTmThxYsrHtdq4e7kv3bp18yMd4D4+ugAO+dH1rFnRBQ66jlSOZcuWFbtd7HGojMtBHqtt27Z5dna2n3baaX7TTTf57NmzE/f96le/8ltuucXd3efNm+fdu3dP3Ne2bVt///33vXfv3u7unpOT4++9955nZWWVquPpp5/2UaNGubt7jx49/Mwzz3R39+HDh/v06dN92bJlPmDAAN+9e7e7u990000+adKkRD3r16/3t99+27Ozs33nzp2+detWP/XUU/2ee+5xd/dzzz3Xb7vtNnd3/8tf/uLnnXeeu7v/4Q9/8JEjRybaMWDAAJ87d25iu/fs2VOqrRs3bnR39507d3pWVpZv2LAhUcfbb7+dstyxY8f6E0884e7umzZt8tNOO823b9/uf/jDH7xly5aJMpNNmTLFr7/++sTtzZs3++7du713796+bt06d3efPHmyX3vtte7ufs011/hzzz13wGV69uzpU6dOdXf3Xbt2+Y4dO3zWrFnev3//RD3Jt7/3ve95fn6+u7vPnDnTs7Oz3d193Lhx3rt3by8sLPT169d7kyZNEo9NZSn5vHd3B+Z7ikxVF4TUWEXDUc6ZM4dZs2YxZMgQ7r77boYPH87QoUM566yzuPfeew97OMoJEyYkhqPctGlTYjjKiRMnMmnSpMRwlAC7du3ihBNOKFZG8nCUwCENR3n22Wdz2223ceWVV3L55ZenHFxo4sSJvPDCCwAUFBTwwQcf0LRp0wPuw1dffZWXX345cUReWFjIJ598AsAFF1xAkyZNSq3TuXNnRo0axejRoxkwYAB9+vRh6dKlLF26lAsuuACAffv2ceKJJxZb7/3330+5zLZt2/j0008ZNGgQAPXq1TtgmyHap88//zwAX//619m4cSNbtmwBoH///tStW5e6detywgknsHbt2pT7qzoogKVGqy3DUY4ZM4b+/fszbdo0evXqxYwZMzj99NMT98+ePZsZM2Ywb948MjIyyMvLo7Cw8ID1FrXt+eefp3379sXmv/XWWxxzzDEp12nXrh0LFixg2rRpjB07ln79+jFo0CCysrJS7ufkulIts3Xr1oO2M1VZJZkZ8O/9CeGH+FQfsNRYtWk4ypUrV9K5c2dGjx5N9+7dWbFiRbHlt2zZQuPGjcnIyGDFihW8+eab5Sr3wgsv5De/+U0i0BYtWnTQtn322WdkZGRw1VVXMWrUKBYuXEj79u1Zv359Ilz37NnDe++9V2y9spY59thjadWqFS+++CIAX375JTt37jzgkJy5ubmJfx6ZPXs2zZo149hjjz1o26ubjoCl2jRv3fqgZy5UtLwDqU3DUc6dO5dZs2aRlpZGx44dufjii4stf9FFF/Hwww/TpUsX2rdvT69evcpV7n/+539yyy230KVLF9ydzMxMXnnllQO2bcmSJdx+++0cddRRpKen89BDD3H00UczZcoUbr75ZrZs2cLevXu55ZZbyMrKSqx3oGWeeOIJbrjhBu644w7S09N57rnn6NKlC3Xq1CE7O5vhw4fTtWvXRFn5+flce+21dOnShYyMDCZNmnTANoei4ShTrFORfSJl03CU5afhKGsODUcp8hWj4ShrJwWwyBFAw1HWTvoSTqqUunOkNqno810BLFWmXr16bNy4USEstYK7s3HjxnKdp1xEXRClpGFmNG/eks8/Xx26MV9prVq1YvXq1axfvz50U0SqRb169Sr0ow4FcCn7gPxK/ffe2io9PZ2TTjopdDNEjljqghARCUQBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogMuSno6ZYWa0aNMmdGtEpAbSeMBl2bMn8U/KlflX6iIiRXQELCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogEVEAlEAi4gEogAWEQlEASwiEogCWEQkEAWwiEggCmARkUAUwCIigSiARUQCUQCLiASiABYRCUQBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogEVEAlEAi4gEogAWEQlEASwiEogCWEQkEAWwiEggCmARkUAUwCIigSiARUQCUQCLiASiABYRCUQBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogEVEAlEAi4gEogAWEQlEASwiEogCWEQkEAWwiEggCmARkUAUwCIigSiARUQCUQCLiASiABYRCaTaAjizdQvMDDMjs3WL6qpWROSIVae6Kvp49Vp8fDRto9ZWV7UiIkcsdUGIiASiABYRCUQBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogEVEAlEAi4gEogAWEQlEASwiEshXJoBbtGiFmdGiRavQTRERqRRfmQBeu/ZTID++FhH56vvKBLCISE2jABYRCUQBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJ5IgO4MzWLTAzzCx0U0REKt0RHcAfr16LjwcfH7olIiKV74gOYBGRmkwBLCISiAJYRCQQBbCISCAKYBGRQBTAIiKBKIBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFAFMAiIoEogEVEAlEAi4gEogAWEQlEASwiEogCWEQkEAWwiEggQQO4RZs2iX89btGmTcimiIhUuzohK19bUACzZkXTffuGbIqISLULdASchpmFqVpE5AgRKID3AflhqhYROULoSzgRkUAUwCIigSiARUQCUQCLiASiABYRCUQBLCISiAJYRCpNixatol+2tmgVuilfCQpgEak0a9d+CuTH13IwCmARkUAUwFIjZbZukRjoKbN1i9DNKZM+stduCuBD9FV5gUwFe8UAAAZNSURBVFeXIy1IPl69Fh8PPh4+Xr2hytp2uM+D2vqRXa+fiAL4EFXXC7y6JL8g0tLqVnh7yhskh/LCO/wX674qa1vx58Hacq2TXE95He7jU943yEOp51C251D2W7JDecM/lH1Q1W8O5u7lX9hsPfBxFbSjGbChCsr9KtE+0D4A7QOomfugrbsfX3JmhQK4qpjZfHfvHrodIWkfaB+A9gHUrn2gLggRkUAUwCIigRwpAfxI6AYcAbQPtA9A+wBq0T44IvqARURqoyPlCFhEpNZRAIuIBBI8gM3sIjN738z+ZWZjQrcnBDNbZWZLzGyxmc0P3Z7qYGaPmdk6M1uaNK+Jmf2vmX0QXzcO2caqVsY+yDezT+PnwmIzuyRkG6uambU2s1lmttzM3jOzH8Tza8VzIWgAm1ka8ABwMdARGGZmHUO2KaC+7p5TW85/BB4HLioxbwww091PA2bGt2uyxym9DwDuj58LOe4+rZrbVN32Aj909w5AL2BknAG14rkQ+gi4J/Avd//Q3XcDk4HLArdJqoG7/x34osTsy4BJ8fQkYGC1NqqalbEPahV3X+PuC+PpbcByoCW15LkQOoBbAgVJt1fH82obB141swVm9t3QjQmoubuvgeiFCZwQuD2hfM/M3o27KGrkR+9UzCwT6Aq8RS15LoQO4FSjd9TG8+LOdvcziLpiRppZbugGSTAPAacAOcAa4N6wzakeZtYAeB64xd23hm5PdQkdwKuB1km3WwGfBWpLMO7+WXy9DniBqGumNlprZicCxNfrAren2rn7Wnff5+77gd9RC54LZpZOFL5PufvUeHateC6EDuC3gdPM7CQzOxoYCrwcuE3VysyOMbOGRdNAP2DpgdeqsV4GromnrwFeCtiWIIpCJzaIGv5csGgMy0eB5e5+X9JdteK5EPyXcPFpNhOANOAxd/9F0AZVMzM7meioF6AO8KfasA/M7Gkgj2jowbXAOOBF4FmgDfAJcIW719gvqcrYB3lE3Q8OrAJuKOoLrYnM7BxgDrAE2B/P/jFRP3CNfy4ED2ARkdoqdBeEiEitpQAWEQlEASwiEogCWEQkEAWwiEggCmApNzPblzRK1+L4p6MVLaORmY2o/NZVLTPLTB61TKQy1AndAPlK2eXuOYdZRiNgBPBgRVYyszR333eYdQdjZnXcfW/odsiRRUfAcljMLM3M7jGzt+MBZG6I5zcws5lmtjAe67holLu7gVPiI+h7zCzPzF5JKu9/zGx4PL3KzO4ws7nAFWZ2iplNjwctmmNmp6doT348iM1sM/vQzG6O5xc7gjWzUWaWH0/PNrP7zezv8bi0PcxsajwW7c+Tiq9jZpPi7ZxiZhnx+t3M7PW4XX9L+gntbDP7LzN7HfhBpe10qTF0BCwVUd/MFsfTH7n7IODbwBZ372FmdYE3zOxVolHuBrn7VjNrBrxpZi8TjevaqehI2szyDlJnobufEy87E7jR3T8wszOJjqK/nmKd04G+QEPgfTN7qBzbttvdc+MBwV8CuhENFbnSzO6Pl2kPfNvd3zCzx4ARZvZr4DfAZe6+3syGAL8ArovXaeTu55ajfqmFFMBSEam6IPoBXcxscHz7OOA0ooGW/ise2W0/0TCjzQ+hzmcgMVrWWcBz0fABANQtY52/uPuXwJdmtq6c9RaNQbIEeK/o579m9iHRgFGbgQJ3fyNe7kngZmA60An437hdaUSjmBVrv0gqCmA5XAZ8393/Vmxm1I1wPNDN3feY2SqgXor191K8K6zkMjvi66OAzeXsg/4yaXof0fP8YPUUrbO/xPr7+ffrpOTv9p1o+99z995ltGVHGfNF1Acsh+1vwE3xkIKYWbt4VLfjgHVx+PYF2sbLbyPqGijyMdDRzOqa2XHAeakqiceI/cjMrojrMTPLrkA71wInmFnTuKtkQAXWLdLGzIqCdhgwF3gfOL5ovpmlm1nWIZQttZACWA7X74FlwML4S67fEh0xPgV0t+hPRq8EVgC4+0aifuKlZnaPuxcQjXr1brzOogPUdSXwbTN7B3iPCvx9lbvvAe4iGmXrlaL2VNBy4BozexdoAjwU/5XWYOC/43YtJuoqETkojYYmIhKIjoBFRAJRAIuIBKIAFhEJRAEsIhKIAlhEJBAFsIhIIApgEZFA/j88VjF3w55i5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(1)\n",
    "plt.clf()\n",
    "\n",
    "X_indices = np.arange(X.shape[-1])\n",
    "\n",
    "# #############################################################################\n",
    "# Univariate feature selection with F-test for feature scoring\n",
    "# We use the default selection function: the 10% most significant features\n",
    "selector = SelectPercentile(f_classif, percentile=10)\n",
    "selector.fit(X, y)\n",
    "scores = -np.log10(selector.pvalues_)\n",
    "scores /= scores.max()\n",
    "plt.bar(X_indices - .45, scores, width=.2,\n",
    "        label=r'Univariate score ($-Log(p_{value})$)', color='darkorange',\n",
    "        edgecolor='black')\n",
    "\n",
    "# #############################################################################\n",
    "# Compare to the weights of an SVM\n",
    "clf = svm.SVC(kernel='linear')\n",
    "clf.fit(X, y)\n",
    "\n",
    "svm_weights = (clf.coef_ ** 2).sum(axis=0)\n",
    "svm_weights /= svm_weights.max()\n",
    "\n",
    "plt.bar(X_indices - .25, svm_weights, width=.2, label='SVM weight',\n",
    "        color='navy', edgecolor='black')\n",
    "\n",
    "clf_selected = svm.SVC(kernel='linear')\n",
    "clf_selected.fit(selector.transform(X), y)\n",
    "\n",
    "svm_weights_selected = (clf_selected.coef_ ** 2).sum(axis=0)\n",
    "svm_weights_selected /= svm_weights_selected.max()\n",
    "\n",
    "plt.bar(X_indices[selector.get_support()] - .05, svm_weights_selected,\n",
    "        width=.2, label='SVM weights after selection', color='c',\n",
    "        edgecolor='black')\n",
    "\n",
    "\n",
    "plt.title(\"Comparing feature selection\")\n",
    "plt.xlabel('Feature number')\n",
    "plt.yticks(())\n",
    "plt.axis('tight')\n",
    "plt.legend(loc='upper right')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
